package com.co.webmagic.task;

import com.co.webmagic.process.AcFunJobProcessor;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.scheduling.annotation.Scheduled;
import org.springframework.stereotype.Component;
import us.codecraft.webmagic.Spider;
import us.codecraft.webmagic.scheduler.BloomFilterDuplicateRemover;
import us.codecraft.webmagic.scheduler.QueueScheduler;

/**
 * @author ldl.plus
 * @date 2020年07月01日  9:47
 */
@Component
public class AcFunDownJob {

    @Autowired
    private AcFunJobProcessor acFunJobProcessor;

    @Scheduled(cron = "0/5 * * * * *")
    public void down() {
        Spider.create(acFunJobProcessor)
                .addUrl(AcFunJobProcessor.searchUrl)
                .setScheduler(new QueueScheduler().setDuplicateRemover(new BloomFilterDuplicateRemover(100000)))
                .thread(1)
                .run();
    }
}
